from os import stat
import re
import time
import pymongo
import requests
from lxml import etree
from pandas.core.frame import DataFrame
# 禁用安全请求警告
from pymongo import MongoClient
from requests.adapters import HTTPAdapter
from urllib3.exceptions import InsecureRequestWarning
import threading
import random
import json
import datetime
import redis
import hashlib


requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
headers = {
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
                  'Chrome/86.0.4240.111 Safari/537.36 ',
}
cookies = [
    {'Cookie': 'SINAGLOBAL=5292419599023.839.1632041533733; ALF=1665133334; SCF=AipZi3nwxW-Rlk_9gqzVTyGoAJ_Iivrfms5dfn4-_8WP5tXiVGsHQo3qj5y8Qjtwi3I6oUOqO--uZ3ELZtuRKak.; wvr=6; _s_tentry=login.sina.com.cn; UOR=,,login.sina.com.cn; Apache=4651444063936.137.1633597338628; ULV=1633597338633:3:1:1:4651444063936.137.1633597338628:1632381437030; SSOLoginState=1633942408; SUB=_2A25MZ4vYDeRhGeNK41AX-CjJyzWIHXVvqxWQrDV8PUJbkNAKLVPlkW1NSQ8yzjsDZ5xRxc0oZ17MdwYogq1PKoI8; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WWF61p6iuMrp-1Un-sSGLhO5NHD95QfShnESoncSK54Ws4Dqcjki--Xi-i2iK.fi--ciK.ciKyhi--ciK.ciKnNP0M4eBtt; webim_unReadCount=%7B%22time%22%3A1634088024043%2C%22dm_pub_total%22%3A3%2C%22chat_group_client%22%3A0%2C%22chat_group_notice%22%3A0%2C%22allcountNum%22%3A14%2C%22msgbox%22%3A0%7D'}
]
request = requests.session()
request.mount('http://', HTTPAdapter(max_retries=3))
request.mount('https://', HTTPAdapter(max_retries=3))
sem = threading.Semaphore(1)


def url_md5(url):
    m = hashlib.md5()
    m.update(url.encode())
    return m.hexdigest()


def redis_connect():
    host = "127.0.0.1"
    port = '6379'
    r = redis.StrictRedis(host=host, port=port, db=0,password=123456)
    return r


# 获取微博内容 微博内容 点赞数 评论数 时间 转发量 链接 微博id
def weibo(url2, f):
    with sem:
        page = 50
        count = 0
        while url2:
            time.sleep(1)
            print(url2)
            if r.exists(url_md5(url2)) == 1:
                print('已经存在了')
                continue
            try:
                resopnse = request.get(url2, headers=headers, verify=False, timeout=10, cookies=random.choice(cookies))
            except requests.exceptions.ConnectionError:
                break
            except requests.exceptions.InvalidURL:
                resopnse = request.get(url2, headers=headers, verify=False, timeout=10, cookies=random.choice(cookies))
            if '抱歉，未找到' in resopnse.text:
                break
            else:
                parser = etree.HTML(resopnse.text)
                all_data = parser.xpath('//*[@class="card"]')
                for data in all_data:
                    dict1 = {}
                    try:
                        refer = data.xpath(
                            './div[1]/div[2]/*[@class="from"]/a')[-1].text
                    except:
                        refer = '暂无来源'
                    name = data.xpath(
                        './div[1]/div[2]/div[1]/div[2]/a[1]')[0].text
                    try:
                        time1 = data.xpath('./div[1]/div[2]/p[2]/a[1]')[0].text.replace('\\n', '').replace(' ',
                                                                                                           '').replace(
                            '\n',
                            '')
                        ff = bool(re.search(r'\d', time1))
                        if ff:
                            time1 = data.xpath('./div[1]/div[2]/p[2]/a[1]')[0].text.replace('\\n', '').replace(' ',
                                                                                                               '').replace(
                                '\n', '')
                        elif '#' in time1:
                            time1 = data.xpath('./div[1]/div[2]/p[3]/a[1]')[0].text.replace('\\n', '').replace(' ',
                                                                                                               '').replace(
                                '\n', '')
                        else:
                            time1 = data.xpath('./div[1]/div[2]/p[3]/a[1]')[0].text.replace('\\n', '').replace(' ',
                                                                                                               '').replace(
                                '\n', '')
                    except:
                        time1 = data.xpath('./div[1]/div[2]/p[3]/a[1]')[0].text.replace('\\n', '').replace(' ',
                                                                                                           '').replace(
                            '\n',
                            '')
                        if '年' in time1:
                            try:
                                time1 = int(time1[0:4])
                            except:
                                time1 = data.xpath('./div[1]/div[2]/p[3]/a[1]')[0].text.replace('\\n', '').replace(' ',
                                                                                                                   '').replace(
                                    '\n', '')
                                time2 = int(time1[0:4])
                                time3 = 2018 - time2
                                if time3 <= 0:
                                    time1 = time1
                                else:
                                    continue
                        else:
                            time1 = time1
                    text = data.xpath('./div[1]/div[2]/p//text()')
                    text = str(text).replace("'\\ue627'", '').replace('\\xa0', '').replace("',", '').replace('\\n',
                                                                                                             '').replace(
                        ' ', '').replace("'", '').replace('hg,', '', 20).replace("'L'", ' ').replace('\\u200b',
                                                                                                     '   ').replace(
                        '[', '').replace(']', '').replace('\\ue40f', '   ').replace('\\ue107', '   ')
                    if '展开全文' in text:
                        text1 = data.xpath('./div[1]/div[2]/p[2]//text()')
                        text1 = str(text1).replace("'\\ue627'", '').replace('\\xa0', '').replace("',", '').replace(
                            '\\n',
                            '').replace(
                            ' ', '').replace("'", '').replace('hg,', '', 20).replace("'L'", ' ').replace('\\u200b',
                                                                                                         '   ').replace(
                            '[',
                            '').replace(
                            ']', '').replace('\\ue40f', '   ').replace('\\ue107', '   ').replace('来自Android客户端',
                                                                                                 '').replace(
                            '来自iPhone客户端', '').replace('来自微博手机版', '').replace('来自微博weibo.com', '')
                        dict1['微博内容'] = text1
                    else:
                        dict1['微博内容'] = text
                    uid = data.xpath(
                        './div[1]/div[1]/a/@href')[0].replace('?refer_flag=1001030103_', '').replace('//weibo.com/', '')
                    zhuanfa = data.xpath(
                        './/*[@class="card-act"]/ul/li[2]/a')[0].text
                    if zhuanfa == ' 转发 ':
                        zhuanfa = ' 转发 0'
                    else:
                        pass
                    pinglun = data.xpath('.//*[@class="card-act"]/ul/li[3]/a')[0].text
                    if pinglun == '评论 ':
                        pinglun = '评论 0'
                    else:
                        pass
                    zan = data.xpath('.//*[@class="card-act"]/ul/li[4]/a/em')[0].text
                    if zan is None:
                        zan = '赞0'
                    else:
                        zan = '赞' + zan
                    hh = data.xpath(
                        './/*[@class="from"]/a[1]/@href')[0]
                    mid = data.xpath(
                        './/*[@class="card-act"]/ul/li[4]/a/@action-data')[0].replace('mid=','')
                    dict1['微博来源'] = refer
                    dict1['用户名'] = name
                    dict1['用户id'] = uid
                    dict1['时间'] = time1
                    dict1['转发量'] = zhuanfa
                    dict1['评论数'] = pinglun
                    dict1['点赞数'] = zan
                    dict1['链接'] = hh
                    dict1['微博id'] = mid
                    r.publish("uid", uid)
                    print(dict1)
                    insertMongo(dict1, 'weibo_content')
                    try:
                        # 获取用户基本信息
                        get_userInfo(uid)
                        # 获取评论
                        comment(mid)
                        # 获取评论
                        transmit(mid)
                        # 获取关注者
                        # attention(uid)
                    except:
                        r.set('start_url',url2)
                    r.set(url_md5(url2), url_md5(url2))
            if '下一页' in resopnse.text:
                try:
                    url2 = 'https://s.weibo.com' + \
                        parser.xpath('//*[@class="next"]/@href')[0]
                except:
                    break
                count = count + 1
                if count >= page:
                    break
            else:
                break


# 获取用户基本信息，如：微博昵称、微博地址、微博头像、关注人数、粉丝数、性别、等级等  id 用户id
def get_userInfo(id):
    dict1 = {}
    url = 'https://m.weibo.cn/api/container/getIndex?type=uid&value=' + id
    print(url)
    data = request.get(url,headers=headers)
    # content = json.loads(data).get('data')
    content = data.json()['data']
    print(content)
    description = content.get('userInfo').get('description')
    profile_url = content.get('userInfo').get('profile_url')
    verified = content.get('userInfo').get('verified')
    guanzhu = content.get('userInfo').get('follow_count')
    name = content.get('userInfo').get('screen_name')
    fensi = content.get('userInfo').get('followers_count')
    gender = content.get('userInfo').get('gender')
    urank = content.get('userInfo').get('urank')
    renzheng = content.get('userInfo').get('verified_reason')
    counts = content.get('userInfo').get('statuses_count')
    dict1['微博昵称'] = name
    dict1['用户id'] = id
    dict1['微博主页地址'] = profile_url
    dict1['是否认证'] = verified
    dict1['微博说明'] = description
    dict1['关注人数'] = guanzhu
    dict1['粉丝数'] = fensi
    dict1['性别'] = gender
    dict1['微博等级'] = urank
    dict1['微博认证'] = renzheng
    dict1['微博数量'] = counts
    '''print("微博昵称：" + name + "\n" + "微博主页地址：" + profile_url + "\n" + "微博头像地址：" + profile_image_url + "\n" + "是否认证：" + str(
        verified) + "\n" + "微博说明：" + description + "\n" + "关注人数：" + str(guanzhu) + "\n" + "粉丝数：" + str(
        fensi) + "\n" + "性别：" + gender + "\n" + "微博等级：" + str(urank) + "\n" + "微博认证：" + renzheng + "\n"
        '微博数量：' + str(counts) + "\n")'''
    print(dict1)
    insertMongo(dict1,'user_info')
    return name, counts


# 评论 获取用户名 评论id mid 性别 评论时间 评论内容 用户id 点赞数 回复数 评论or回复 mid=mid 
# https://m.weibo.cn/comments/hotflow?id=4661442680723401&mid=4661442680723401&max_id_type=0
# https://m.weibo.cn/comments/hotFlowChild?cid=4675635052548577&max_id=0&max_id_type=0    具体评论url
# https://m.weibo.cn/comments/hotflow?id=4691172646455472&mid=4691172646455472&max_id=150124914358992
def comment(mid):
    url = 'https://m.weibo.cn/comments/hotflow?id={}&mid={}&max_id_type=0'.format(mid,mid)
    print(url)
    res = request.get(url)
    if res.json()['ok'] == 0:
        return
    results = res.json()['data']['data']
    max_id = res.json()['data']['max_id']
    for result in results:
        dict1 = {}
        dict1['用户名'] = result['user']['screen_name']
        dict1['评论id'] = result['bid']
        dict1['mid'] = result['mid']
        dict1['性别'] = result['user']['gender']
        dict1['评论时间'] = result['created_at']
        dict1['评论内容'] = result['text']
        dict1['用户id'] = result['user']['id']
        dict1['点赞数'] = result['like_count']
        dict1['回复数'] = result['total_number']
        dict1['评论or回复'] = '评论'
        insertMongo(dict1, 'weibo_comment')
        if result['comments']!=False:
            cid = result['mid']
            max_id = 0
            while True:
                url = 'https://m.weibo.cn/comments/hotFlowChild?cid={}&max_id={}&max_id_type=0'.format(
                    cid, max_id)
                print(url)
                ress = request.get(url,headers=headers)
                for comment in ress.json()['data']:
                    dict2 = {}
                    dict2['用户名'] = comment['user']['screen_name']
                    dict2['评论id'] = comment['bid']
                    dict2['mid'] = comment['mid']
                    dict2['性别'] = comment['user']['gender']
                    dict2['评论时间'] = comment['created_at']
                    dict2['评论内容'] = comment['text']
                    dict2['用户id'] = comment['user']['id']
                    dict2['点赞数'] = result['like_count']
                    dict2['回复数'] = 0
                    dict2['评论or回复'] = '回复'
                    insertMongo(dict2, 'weibo_comment')
                if ress.json()['max_id'] == 0:
                    break
                else:
                    max_id = ress.json()['max_id']
    while True:
        if max_id != 0:
            url = 'https://m.weibo.cn/comments/hotflow?id={}&mid={}&max_id_type=0&max_id={}'.format(
                mid, mid,max_id)
            try:
                res = request.get(url, headers=headers)
                results = res.json()['data']['data']
            except:
                try:
                    res = request.get(url, headers=headers)
                    results = res.json()['data']['data']
                except:
                    break
            max_id = res.json()['data']['max_id']
            for result in results:
                dict1 = {}
                dict1['用户名'] = result['user']['screen_name']
                dict1['评论id'] = result['bid']
                dict1['mid'] = result['mid']
                dict1['性别'] = result['user']['gender']
                dict1['评论时间'] = result['created_at']
                dict1['评论内容'] = result['text']
                dict1['用户id'] = result['user']['id']
                dict1['点赞数'] = result['like_count']
                dict1['回复数'] = result['total_number']
                dict1['评论or回复'] = '评论'
                insertMongo(dict1, 'weibo_comment')
                if result['comments'] != False:
                    cid = result['mid']
                    max_id = 0
                    while True:
                        url = 'https://m.weibo.cn/comments/hotFlowChild?cid={}&max_id={}&max_id_type=0'.format(
                            cid, max_id)
                        ress = request.get(url, headers=headers)
                        for comment in ress['data']:
                            dict2 = {}
                            dict2['用户名'] = comment['user']['screen_name']
                            dict2['评论id'] = comment['bid']
                            dict2['mid'] = comment['mid']
                            dict2['性别'] = comment['user']['gender']
                            dict2['评论时间'] = comment['created_at']
                            dict2['评论内容'] = comment['text']
                            dict2['用户id'] = comment['user']['id']
                            dict2['点赞数'] = result['like_count']
                            dict2['回复数'] = 0
                            dict2['评论or回复'] = '回复'
                            insertMongo(dict2, 'weibo_comment')
                        if ress['max_id'] == 0:
                            break
                        else:
                            max_id = ress['max_id']
        else:
            return


# 转发  获取用户名 转发id  mid 性别 评论时间 评论内容 用户id 发布来源 转发  id=mid
def transmit(id):
    url = 'https://m.weibo.cn/api/statuses/repostTimeline?id={}&page={}'.format(id,1)
    print(url)
    res = request.get(url,headers=headers)
    if res.json()['msg'] == '还没有人转发过':
        return
    results = res.json()['data']['data']
    for result in results:
        dict1 = {}
        dict1['用户名'] = result['user']['screen_name']
        dict1['转发id'] = result['bid']
        dict1['mid'] = result['mid']
        dict1['性别'] = result['user']['gender']
        dict1['评论时间'] = result['created_at']
        dict1['评论内容'] = result['text']
        dict1['用户id'] = result['user']['id']
        dict1['发布来源'] = result['source']
        dict1['转发'] = '转发'
        insertMongo(dict1, 'weibo_transmit')
    max = res.json()['data']['max']
    if max ==1:
        return
    else:
        for i in range(2,max):
            url = 'https://m.weibo.cn/api/statuses/repostTimeline?id={}&page={}'.format(
                id, i)
            res = request.get(url, headers=headers)
            results = res.json()['data']['data']
            for result in results:
                dict1 = {}
                dict1['用户名'] = result['user']['screen_name']
                dict1['转发id'] = result['bid']
                dict1['mid'] = result['mid']
                dict1['性别'] = result['user']['gender']
                dict1['评论时间'] = result['created_at']
                dict1['评论内容'] = result['text']
                dict1['用户id'] = result['user']['id']
                dict1['发布来源'] = result['source']
                dict1['转发'] = '转发'
                insertMongo(dict1, 'weibo_transmit')


# uid 用户id
# 关注  获取关注者的id 被关注者的id  爬取时间
# https://m.weibo.cn/api/container/getIndex?containerid=231051_-_fans_-_1651428902
# https://m.weibo.cn/api/container/getIndex?containerid=231051_-_fans_-_2615577972
# https://m.weibo.cn/api/container/getIndex?containerid=231051_-_fans_-_1651428902&since_id=22
def attention(uid):
    url = 'https://m.weibo.cn/api/container/getIndex?containerid=231051_-_fans_-_{}'.format(uid)
    print(url)
    res = request.get(url,headers=headers)
    print(len(res.json()['data']['cards']))
    if len(res.json()['data']['cards']) == 0:
        res = request.get(url, headers=headers,timeout=10)
    if len(res.json()['data']['cards']) == 1:
        for data in res.json()['data']['cards'][0]['card_group']:
            dict1 = {}
            da = datetime.datetime.strptime(
                str(datetime.date.today()), '%Y-%m-%d')
            todays = datetime.datetime.strftime(da, '%Y-%m-%d')
            try:
                dict1['关注者id'] = data['user']['id']
            except:
                continue
            dict1['被关注者id'] = uid
            dict1['抓取时间'] = todays
            insertMongo(dict1, 'weibo_attention')
            print(
                '==================================插入数据成功======================================')
    else:
        for data in res.json()['data']['cards'][-1]['card_group']:
            dict1 = {}
            da = datetime.datetime.strptime(str(datetime.date.today()), '%Y-%m-%d')
            todays = datetime.datetime.strftime(da, '%Y-%m-%d')
            dict1['关注者id'] = data['user']['id']
            dict1['被关注者id'] = uid
            dict1['抓取时间'] = todays
            insertMongo(dict1, 'weibo_attention')
            print(
                '==================================插入数据成功======================================')
    try:
        since_id = res.json()['data']['cardlistInfo']['since_id']
    except:
        return
    try:
        while True:
            url = 'https://m.weibo.cn/api/container/getIndex?containerid=231051_-_fans_-_{}&since_id={}'.format(
                uid, since_id)
            res = request.get(url, headers=headers)
            since_id = res.json()['data']['cardlistInfo']['since_id']
            for data in res.json()['data']['cards'][0]['card_group']:
                dict1 = {}
                da = datetime.datetime.strptime(str(datetime.date.today()), '%Y-%m-%d')
                todays = datetime.datetime.strftime(da, '%Y-%m-%d')
                dict1['关注者id'] = data['user']['id']
                dict1['被关注者id'] = uid
                dict1['抓取时间'] = todays
                print('==================================插入数据成功======================================')
                insertMongo(dict1, 'weibo_attention')
    except Exception as e:
        print('报错了，原因是',e)
        return


def insertMongo(dict,table):
    """插入mongodb数据库表格信息"""
    client = MongoClient("mongodb://192.168.0.10:27017/")
    collection = client['weibo'][table]
    collection.insert_one(dict)


if __name__ == "__main__":
    r = redis_connect()
    # if r.exists('start_url') == 1:
    #     start_url = r.get('start_url')
    print('程序默认最大页数是50页')
    month = int(input('请输入月份:'))
    if month < 10:
        month = str(month)
        month = '0' + month
    else:
        month = month
    tty = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]
    llu = ['国际快递']
    list1 = []
    f = open('./weibodata2.txt','a+',encoding='utf-8')
    for rre in llu:
        for i in range(1, 10):
            for u in tty:
                url5 = 'https://s.weibo.com/weibo?q={}&typeall=1&suball=1&timescope=custom:2021-{}-0{}-{}:2021-{}-0{}-{}&Refer=g'.format(
                        rre, month, i, u, month, i, u + 1)
                weibo(url5,f)
    for rre in llu:
        for i in range(10, 31):
            for u in tty:
                url5 = 'https://s.weibo.com/weibo?q={}&typeall=1&suball=1&timescope=custom:2021-{}-{}-{}:2021-{}-{}-{}&Refer=g'.format(
                    rre, month, i, u, month, i, u + 1)
                weibo(url5, f)
